import java.io.FileWriter
import java.nio.file.Paths
import java.util
import java.util.UUID

import org.slf4j.LoggerFactory

import scala.util.Try

/**
  * Created by kylin on 17-4-17.
  */
object Convert {

  val log = LoggerFactory.getLogger(this.getClass)

  def main(args: Array[String]): Unit = {

    log.info("loading entity id mapping...")

    //    val eid = new java.util.concurrent.ConcurrentHashMap[String, String](34930739)
    //    val notFound = java.util.Collections.synchronizedList(new util.ArrayList[(String, String)](1000000))

    val eid = io.Source.fromFile("%s/zjh/eid.csv".format(System.getProperty("user.home")))
      .getLines()
      .toList
      .par
      .map(ln => {
        try {
          var i1 = 0
          var i2 = 0
          var i3 = 0
          var i4 = 0
          if (ln.charAt(0) == '"') {
            i1 = 1
            i2 = ln.indexOf('"', i1)
            i3 = i2 + 2
          } else {
            i2 = ln.indexOf(',')
            i3 = i2 + 1
          }
          if (ln.charAt(i3) == '"') {
            i3 += 1
            i4 = ln.lastIndexOf('"')
          } else {
            i4 = ln.length
          }
          //          eid.put(ln.substring(i1, i2), ln.substring(i3, i4))
          val e = ln.substring(i1, i2)
          val d = ln.substring(i3, i4)
          (e, d)
        } catch {
          case e: Exception =>
            log.error(s"err load mapping line: $ln")
            System.exit(-2)
            null
        }
      })
      .toMap.withDefault(_ => s"other-${UUID.randomUUID().toString}")

    log.info(s"done loading entity id mapping, size: ${eid.size}")

    log.info("clearing mem...")
    log.info(s"before gc free mem: ${Runtime.getRuntime.freeMemory() / 1000000}M")
    System.gc()   // free about 1G mem
    log.info(s"after gc free mem: ${Runtime.getRuntime.freeMemory() / 1000000}M")
    log.info("done clearing mem")


    //    val files = args.sortBy(f => Try(new java.io.File(f).length()).getOrElse(0L)).toList

    val files = List("instance_types_transitive_en.ttl", "interlanguage_links_en.ttl", "wikipedia_links_en-rel.ttl")

    log.info(s"files to process: ${files.mkString(" ")}")

    val blocks = Map("wikipedia_links_en-rel.ttl" -> List(
      (0, 10000000), (10000000, 20000000), (20000000, 30000000), (30000000, 40000000)),
      "interlanguage_links_en.ttl" -> List(
        (0, 10000000), (10000000, 20000000), (20000000, 30000000), (30000000, 40000000)
      ),
      "instance_types_transitive_en.ttl" -> List(
        (0, 10000000), (10000000, 20000000), (20000000, 30000000), (30000000, 40000000)
      ))


    files
      .foreach(f => {
        try {
          insertFile(f)
        } catch {
          case e: Exception =>
            log.error(s"err file: $f", e)
        }
      })

    def insertBlock(f: String, slice: (Int, Int), idx: Int): Unit = {
      log.info(s"block: $idx, slice: ${slice.toString()}, file: $f")

      val b = f.substring(0, f.indexOf('.'))
      val w = new FileWriter(s"${System.getProperty("user.home")}/zjh/rel-mysql2/entity_relation_libs_dbpedia.$b-$idx")

      io.Source.fromFile(s"${System.getProperty("user.home")}/zjh/ttl-rel/$f")
        .getLines().slice(slice._1, slice._2)
        .filter(_.charAt(0) != '#')
        .toList
        .foreach(ln => {
          try {
            val a = ln.split(" ")
            if (a.length == 0) {
              log.warn(s"unexpected line: $ln, in file: $f")
              null
            } else {
              val s = a(0)
              val p = a(1)
              val o = a(2)
              val sid = eid(s)
              val oid = eid(o)
              val mysqlid = UUID.randomUUID().toString.replace("-", "")
              w.write(s"${mysqlid}\t$sid\t$p\t$oid\n")
            }
          } catch {
            case e: Exception =>
              log.warn(s"err line: $ln, block: $idx, slice: ${slice.toString()}, file: $f")
          }
        })

      w.close()

//      log.info("clearing mem...")
//      log.info(s"before gc free mem: ${Runtime.getRuntime.freeMemory() / 1000000}M")
//      System.gc()
//      log.info(s"after gc free mem: ${Runtime.getRuntime.freeMemory() / 1000000}M")
//      log.info("done clearing mem")
    }

    def insertFile(f: String): Unit = {
      log.info(s"process file: $f")

      blocks(f)
        .zipWithIndex
        .par
        .foreach(b => {
          try {
            insertBlock(f, b._1, b._2)
          }
          catch {
            case e: Exception =>
              log.info(s"block: ${b._2}, slice: ${b._1.toString()}, file: $f")
          }
        })

      log.info(s"done process file: $f")
    }

    log.info("done main")
  }
}
